% run a clustering experiment
% varargin
% 'algorithm'            : name of algorithm
% 'average'              : number of experments used for averaging
% 'criteria'             : name of criteria used to compare
%                          clustering assignment
% 'k'                    : list of number of clusters
% 'column'               : is column-wise data
% 'test_class'           : repeat using test_class from old experiment
% 'output'               : result output

% example
% test_class = generateTestClass(gnd,'k',5:5:30,'average',10);
% [criteria,test_class] = clusteringExperiment(Y,gnd,'algorithm', 'multiplicativeNMFClustering', 'k',5:5:10,'test_class', test_class,'output','../result/tdt2_mulnmf_cluster');

function [criteria, test_class] = clusteringExperiment(fea, gnd, varargin)
    par.algorithm = 'multiplicativeNMFClustering';
    par.average = 10;
    par.criteria = 'normalizedMutualInformation';
    par.k = 5;
    par.column = false;
    par.test_class = {};
    par.output = '';
    par = process_parameter(par, varargin{:});
    display('Running Clustering algorithm');
    display(par);
    
    if par.column, fea = fea'; gnd = gnd'; end

    class = unique(gnd);
    n_class = length(class);
    test_class = par.test_class;

    for i=1:length(par.k)
        k = par.k(i);
        sum_criteria = 0;
        for j = 1:par.average
            % pick randomly k classes
            if k <= size(test_class,1) && j <= size(test_class,2) ...
                    && length(test_class{k,j}) == k
                display('reused');
                chosen_class = test_class{k,j};
            else
                display('create');
                chosen_class = class(randCombination(n_class, k))';
                test_class{k,j} = chosen_class;
            end
            
            display(sprintf('k=%d j=%d', k,j));
            display(chosen_class);
            X = []; y = [];
            for t = 1:length(chosen_class), 
                X = [X; fea(gnd == chosen_class(t),:)]; 
                y = [y; gnd(gnd == chosen_class(t),:)];
            end
            I = randperm(size(X,1));
            X = X(I,:);
            y = y(I);
            
            % run clustering algorithm on X
            str = [par.algorithm '(X, varargin{:}, ''k'', k, ' ...
                                '''column'', false)'];
            cluster = eval(str);
            
            % computer clustering comparison criteria
            str = [par.criteria '(y, cluster)'];
            criteria_val = eval(str);
            
            sum_criteria = sum_criteria + criteria_val;
        end
        criteria(i) = sum_criteria / par.average;
        result.(par.criteria) = criteria;
        if ~isempty(par.output)
            save(par.output, 'par', 'result')
        end
    end
    
end
